{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "source": [
    "import torch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import random\n",
    "import pandas as pd\n",
    "from common.configs.path import paths\n",
    "from common.configs.tools import label_map\n",
    "from collections import Counter\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from torch.utils.data import (TensorDataset, DataLoader, RandomSampler,\n",
    "                              SequentialSampler)\n",
    "\n",
    "def data_loader(train_inputs, val_inputs, train_labels, val_labels,\n",
    "                batch_size=50):\n",
    "    \"\"\"Convert train and validation sets to torch.Tensors and load them to\n",
    "    DataLoader.\n",
    "    \"\"\"\n",
    "\n",
    "    # Convert data type to torch.Tensor\n",
    "    train_inputs, val_inputs, train_labels, val_labels =\\\n",
    "    tuple(torch.tensor(data) for data in\n",
    "          [train_inputs, val_inputs, train_labels, val_labels])\n",
    "\n",
    "    # Specify batch_size\n",
    "    batch_size = batch_size\n",
    "\n",
    "    # Create DataLoader for training data\n",
    "    train_data = TensorDataset(train_inputs, train_labels)\n",
    "    train_sampler = RandomSampler(train_data)\n",
    "    train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "\n",
    "    # Create DataLoader for validation data\n",
    "    val_data = TensorDataset(val_inputs, val_labels)\n",
    "    val_sampler = SequentialSampler(val_data)\n",
    "    val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)\n",
    "\n",
    "    return train_dataloader, val_dataloader"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "source": [
    "train_df = pd.read_csv(paths['train_data'])\n",
    "test_df = pd.read_csv(paths['test_data'])\n",
    "train_df.label = train_df.label.apply(lambda e: label_map[e])\n",
    "Y = train_df.label.values\n",
    "\n",
    "train_pre = torch.load(r'pretrain/encoded/train_tune_vectors.pt').numpy()\n",
    "test_pre = torch.load(r'pretrain/encoded/test_tune_vectors.pt').numpy()\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "source": [
    "oversample = SMOTE()\n",
    "X, y = oversample.fit_resample(train_pre, Y)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "source": [
    "if torch.cuda.is_available():\n",
    "    print('gpu is available: {}'.format(torch.cuda.get_device_name(0)))\n",
    "    print('device count: {}'.format(torch.cuda.device_count()))\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print('Using device:', device)\n",
    "print()\n",
    "\n",
    "train_inputs, val_inputs, train_labels, val_labels = train_test_split(\n",
    "    X, y, test_size=0.1, random_state=42)\n",
    "\n",
    "train_dataloader, val_dataloader = data_loader(train_inputs, val_inputs, train_labels, val_labels, batch_size=50)\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "gpu is available: NVIDIA GeForce GTX 1050 with Max-Q Design\n",
      "device count: 1\n",
      "Using device: cuda\n",
      "\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "source": [
    "filter_sizes = [2, 3, 4]\n",
    "num_filters = [2, 2, 2]\n",
    "\n",
    "class TextCNN(nn.Module):\n",
    "    \"\"\"An 1D Convulational Neural Network for Sentence Classification.\"\"\"\n",
    "    def __init__(self,\n",
    "                 pretrained_embedding=None,\n",
    "                 filter_sizes=[3, 4, 5],\n",
    "                 num_filters=[100, 100, 100],\n",
    "                 num_classes=35,\n",
    "                 dropout=0.5):\n",
    "        \"\"\"\n",
    "        The constructor for CNN_NLP class.\n",
    "\n",
    "        Args:\n",
    "            pretrained_embedding (torch.Tensor): Pretrained embeddings with\n",
    "                shape (vocab_size, embed_dim)\n",
    "            freeze_embedding (bool): Set to False to fine-tune pretraiend\n",
    "                vectors. Default: False\n",
    "            vocab_size (int): Need to be specified when not pretrained word\n",
    "                embeddings are not used.\n",
    "            embed_dim (int): Dimension of word vectors. Need to be specified\n",
    "                when pretrained word embeddings are not used. Default: 300\n",
    "            filter_sizes (List[int]): List of filter sizes. Default: [3, 4, 5]\n",
    "            num_filters (List[int]): List of number of filters, has the same\n",
    "                length as `filter_sizes`. Default: [100, 100, 100]\n",
    "            n_classes (int): Number of classes. Default: 2\n",
    "            dropout (float): Dropout rate. Default: 0.5\n",
    "        \"\"\"\n",
    "\n",
    "        super(TextCNN, self).__init__()\n",
    "        # Embedding layer\n",
    "        self.fc1 = nn.Linear(64, 128)\n",
    "\n",
    "        # Conv Network\n",
    "        self.conv1d_list = nn.ModuleList([\n",
    "            nn.Conv1d(in_channels=1,\n",
    "                      out_channels=num_filters[i],\n",
    "                      kernel_size=filter_sizes[i])\n",
    "            for i in range(len(filter_sizes))\n",
    "        ])\n",
    "        # Fully-connected layer and Dropout\n",
    "        self.fc = nn.Linear(np.sum(num_filters), num_classes)\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "    def forward(self, input_ids):\n",
    "        \"\"\"Perform a forward pass through the network.\n",
    "\n",
    "        Args:\n",
    "            input_ids (torch.Tensor): A tensor of token ids with shape\n",
    "                (batch_size, max_sent_length)\n",
    "\n",
    "        Returns:\n",
    "            logits (torch.Tensor): Output logits with shape (batch_size,\n",
    "                n_classes)\n",
    "        \"\"\"\n",
    "\n",
    "        # Get embeddings from `input_ids`. Output shape: (b, max_len, embed_dim)\n",
    "        x_embed = self.fc(input_ids).float()\n",
    "\n",
    "        # Permute `x_embed` to match input shape requirement of `nn.Conv1d`.\n",
    "        # Output shape: (b, embed_dim, max_len)\n",
    "        x_reshaped = x_embed.permute(0, 2, 1)\n",
    "\n",
    "        # Apply CNN and ReLU. Output shape: (b, num_filters[i], L_out)\n",
    "        x_conv_list = [F.relu(conv1d(x_reshaped)) for conv1d in self.conv1d_list]\n",
    "\n",
    "        # Max pooling. Output shape: (b, num_filters[i], 1)\n",
    "        x_pool_list = [F.max_pool1d(x_conv, kernel_size=x_conv.shape[2])\n",
    "            for x_conv in x_conv_list]\n",
    "        \n",
    "        # Concatenate x_pool_list to feed the fully connected layer.\n",
    "        # Output shape: (b, sum(num_filters))\n",
    "        x_fc = torch.cat([x_pool.squeeze(dim=2) for x_pool in x_pool_list],\n",
    "                         dim=1)\n",
    "        \n",
    "        # Compute logits. Output shape: (b, n_classes)\n",
    "        logits = self.fc(self.dropout(x_fc))\n",
    "\n",
    "        return logits"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "source": [
    "# Specify loss function\n",
    "import time\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "def initilize_model(filter_sizes=[3, 4, 5],\n",
    "                    num_filters=[100, 100, 100],\n",
    "                    num_classes=35,\n",
    "                    dropout=0.5,\n",
    "                    learning_rate=0.01):\n",
    "    \"\"\"Instantiate a CNN model and an optimizer.\"\"\"\n",
    "\n",
    "    assert (len(filter_sizes) == len(num_filters)), \"filter_sizes and \\\n",
    "    num_filters need to be of the same length.\"\n",
    "\n",
    "    # Instantiate CNN model\n",
    "    cnn_model = TextCNN(filter_sizes=filter_sizes,\n",
    "                        num_filters=num_filters,\n",
    "                        num_classes=35,\n",
    "                        dropout=0.5)\n",
    "    \n",
    "    # Send model to `device` (GPU/CPU)\n",
    "    cnn_model.to(device)\n",
    "\n",
    "    # Instantiate Adadelta optimizer\n",
    "    optimizer = optim.Adadelta(cnn_model.parameters(),\n",
    "                               lr=learning_rate,\n",
    "                               rho=0.95)\n",
    "\n",
    "    return cnn_model, optimizer\n",
    "\n",
    "def set_seed(seed_value=42):\n",
    "    \"\"\"Set seed for reproducibility.\"\"\"\n",
    "\n",
    "    random.seed(seed_value)\n",
    "    np.random.seed(seed_value)\n",
    "    torch.manual_seed(seed_value)\n",
    "    torch.cuda.manual_seed_all(seed_value)\n",
    "\n",
    "def train(model, optimizer, train_dataloader, val_dataloader=None, epochs=50):\n",
    "    \"\"\"Train the CNN model.\"\"\"\n",
    "    \n",
    "    # Tracking best validation accuracy\n",
    "    best_accuracy = 0\n",
    "\n",
    "    # Start training loop\n",
    "    print(\"Start training...\\n\")\n",
    "    print(f\"{'Epoch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
    "    print(\"-\"*60)\n",
    "\n",
    "    for epoch_i in range(epochs):\n",
    "        # =======================================\n",
    "        #               Training\n",
    "        # =======================================\n",
    "\n",
    "        # Tracking time and loss\n",
    "        t0_epoch = time.time()\n",
    "        total_loss = 0\n",
    "\n",
    "        # Put the model into the training mode\n",
    "        model.train()\n",
    "\n",
    "        for step, batch in enumerate(train_dataloader):\n",
    "            # Load batch to GPU\n",
    "            b_input_ids, b_labels = tuple(t.to(device) for t in batch)\n",
    "\n",
    "            # Zero out any previously calculated gradients\n",
    "            model.zero_grad()\n",
    "\n",
    "            # Perform a forward pass. This will return logits.\n",
    "            logits = model(torch.Tensor(b_input_ids))\n",
    "\n",
    "            # Compute loss and accumulate the loss values\n",
    "            loss = loss_fn(logits, b_labels)\n",
    "            total_loss += loss.item()\n",
    "\n",
    "            # Perform a backward pass to calculate gradients\n",
    "            loss.backward(retain_graph=True)\n",
    "\n",
    "            # Update parameters\n",
    "            optimizer.step()\n",
    "\n",
    "        # Calculate the average loss over the entire training data\n",
    "        avg_train_loss = total_loss / len(train_dataloader)\n",
    "\n",
    "        # =======================================\n",
    "        #               Evaluation\n",
    "        # =======================================\n",
    "        if val_dataloader is not None:\n",
    "            # After the completion of each training epoch, measure the model's\n",
    "            # performance on our validation set.\n",
    "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
    "\n",
    "            # Track the best accuracy\n",
    "            if val_accuracy > best_accuracy:\n",
    "                best_accuracy = val_accuracy\n",
    "\n",
    "            # Print performance over the entire training data\n",
    "            time_elapsed = time.time() - t0_epoch\n",
    "            print(f\"{epoch_i + 1:^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
    "            \n",
    "    print(\"\\n\")\n",
    "    print(f\"Training complete! Best accuracy: {best_accuracy:.2f}%.\")\n",
    "\n",
    "def evaluate(model, val_dataloader):\n",
    "    \"\"\"After the completion of each training epoch, measure the model's\n",
    "    performance on our validation set.\n",
    "    \"\"\"\n",
    "    # Put the model into the evaluation mode. The dropout layers are disabled\n",
    "    # during the test time.\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables\n",
    "    val_accuracy = []\n",
    "    val_loss = []\n",
    "\n",
    "    # For each batch in our validation set...\n",
    "    for batch in val_dataloader:\n",
    "        # Load batch to GPU\n",
    "        b_input_ids, b_labels = tuple(t.to(device) for t in batch)\n",
    "\n",
    "        # Compute logits\n",
    "        with torch.no_grad():\n",
    "            logits = model(b_input_ids)\n",
    "\n",
    "        # Compute loss\n",
    "        loss = loss_fn(logits, b_labels)\n",
    "        val_loss.append(loss.item())\n",
    "\n",
    "        # Get the predictions\n",
    "        preds = torch.argmax(logits, dim=1).flatten()\n",
    "\n",
    "        # Calculate the accuracy rate\n",
    "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
    "        val_accuracy.append(accuracy)\n",
    "\n",
    "    # Compute the average accuracy and loss over the validation set.\n",
    "    val_loss = np.mean(val_loss)\n",
    "    val_accuracy = np.mean(val_accuracy)\n",
    "\n",
    "    return val_loss, val_accuracy"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "set_seed(42)\n",
    "cnn_rand, optimizer = initilize_model(learning_rate=0.25,\n",
    "                                      dropout=0.5)\n",
    "train(cnn_rand, optimizer, train_dataloader, val_dataloader, epochs=50)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import CategoricalNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import pickle"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "source": [
    "mlp = MLPClassifier(random_state=31)\n",
    "linsvm = LinearSVC(random_state=31, multi_class=\"ovr\")\n",
    "etc = ExtraTreesClassifier(n_estimators=100, min_samples_split=2, random_state=31)\n",
    "ada = AdaBoostClassifier(n_estimators=100, random_state=31)\n",
    "gb = GradientBoostingClassifier(random_state=31, n_estimators=100)\n",
    "dt = DecisionTreeClassifier(random_state=31)\n",
    "svc = SVC()\n",
    "rf = RandomForestClassifier(max_depth=5, \n",
    "                            random_state=31,\n",
    "                            n_jobs=-1)\n",
    "cnb = CategoricalNB()\n",
    "lr = LogisticRegression(random_state=31,\n",
    "                        multi_class='ovr', n_jobs=-1)\n",
    "knn = KNeighborsClassifier(n_neighbors=10, n_jobs=-1)\n",
    "lgbm = lgb.LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
    "                         importance_type='split', learning_rate=0.01, max_depth=-1,\n",
    "                         min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
    "                         n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
    "                         random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
    "                         subsample=1.0, subsample_for_bin=2000, subsample_freq=0)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "source": [
    "# X = input_ids\n",
    "# y = labels\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=31)\n",
    "\n",
    "\n",
    "for k, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "\n",
    "    eclf1 = VotingClassifier(estimators=[('gb', gb), ('mlp', mlp),\n",
    "                                         ('linsvm', linsvm),\n",
    "                                         ('ada', ada),\n",
    "                                         ('etc', etc),\n",
    "                                         ('dt', dt),\n",
    "                                         ('svc', svc),\n",
    "                                         ('rf', rf), \n",
    "                                         ('cnb', cnb), \n",
    "                                         ('lr', lr), \n",
    "                                         ('knn', knn),\n",
    "                                         ('lgb', lgbm)\n",
    "                                         ],\n",
    "                             n_jobs=-1, verbose=True)\n",
    "    eclf1 = eclf1.fit(X_train, y_train)\n",
    "    y_pred = eclf1.predict(X_test)\n",
    "    f1 = f1_score(y_test, y_pred, average='macro')\n",
    "    print('[{}] f1: {}'.format(k, f1))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit (conda)"
  },
  "interpreter": {
   "hash": "1b89aa55be347d0b8cc51b3a166e8002614a385bd8cff32165269c80e70c12a7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}